import random

import redis
import requests

def should_crawl(url):
    client = redis.Redis(host='localhost', port=6379, db=0)
    urls = client.lrange('已爬过的地址', 0, -1)
    return url.encode() in urls


def save_url(url):
    client = redis.Redis(host='localhost', port=6379, db=0)
    client.lpush('已爬过的地址', url)
    client.close()

def main():
    urls = [f"https://httpbin.org/get?age={random.randint(1,100)}" for i in range(10)]
    urls = set(urls)
    for url in urls:
        if not should_crawl(url):
            res = requests.get(url)
            print(res,url)
            save_url(url)
        else:
            print('已爬过不需要再爬', url)


if __name__ == '__main__':
    main()






